{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install pytorch-pretrained-bert"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',\n",
    "                    datefmt='%m/%d/%Y %H:%M:%S',\n",
    "                    level=logging.INFO)\n",
    "logger = logging.getLogger(__name__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11/07/2019 12:11:51 - INFO - pytorch_pretrained_bert.modeling -   Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex .\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab_file:  https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11/07/2019 12:11:52 - INFO - pytorch_pretrained_bert.tokenization -   loading vocabulary file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt from cache at /home/sam/.pytorch_pretrained_bert/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from pytorch_pretrained_bert import BertTokenizer, BertModel, BertForMaskedLM\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "BERT_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data shape = (566, 3) \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idx</th>\n",
       "      <th>tweets</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>768744566203768832</td>\n",
       "      <td>earthquake italy 267 dead hundreds injured</td>\n",
       "      <td>[2]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>768822900812046340</td>\n",
       "      <td>dlusvideonews montepulciano damages earthquake...</td>\n",
       "      <td>[2]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>768848577225428992</td>\n",
       "      <td>italy quake young girl found alive rubble leas...</td>\n",
       "      <td>[0, 3]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>769138472636469248</td>\n",
       "      <td>new post national disaster italy quake death t...</td>\n",
       "      <td>[2]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>769113679073800193</td>\n",
       "      <td>number dead italy quake climbs first funerals ...</td>\n",
       "      <td>[2]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  idx                                             tweets  \\\n",
       "0  768744566203768832        earthquake italy 267 dead hundreds injured    \n",
       "1  768822900812046340  dlusvideonews montepulciano damages earthquake...   \n",
       "2  768848577225428992  italy quake young girl found alive rubble leas...   \n",
       "3  769138472636469248  new post national disaster italy quake death t...   \n",
       "4  769113679073800193  number dead italy quake climbs first funerals ...   \n",
       "\n",
       "   labels  \n",
       "0     [2]  \n",
       "1     [2]  \n",
       "2  [0, 3]  \n",
       "3     [2]  \n",
       "4     [2]  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "from os.path import join, exists\n",
    "from collections import OrderedDict\n",
    "\n",
    "datafile = \"~/Downloads/smerp_labeled_validation.csv\"\n",
    "\n",
    "def load_json(filename: str,filepath: str = '',date_time_tag: str = '',ext: str = \".json\",\n",
    "              show_path: bool = False) -> OrderedDict:\n",
    "    file_loc = join(filepath,date_time_tag + filename + ext)\n",
    "    if show_path:\n",
    "        print(\"Reading JSON file: [{}]\".format(file_loc))\n",
    "    if exists(join(filepath,date_time_tag + filename + ext)):\n",
    "        try:\n",
    "            with open(file_loc, encoding=\"utf-8\") as file:\n",
    "                json_dict = json.load(file)\n",
    "                json_dict = OrderedDict(json_dict)\n",
    "                # json_dict = OrderedDict(json.load(file))\n",
    "            file.close()\n",
    "            return json_dict\n",
    "        except Exception as e:\n",
    "            print(\"Could not open file as JSON: [{}]. \\n Reason:[{}]\".format(file_loc,e))\n",
    "            with open(file_loc, encoding=\"utf-8\") as file:\n",
    "                json_dict = str(file)\n",
    "                json_dict = json.loads(json_dict)\n",
    "                # json_dict = OrderedDict(json_dict)\n",
    "            return json_dict\n",
    "    else:\n",
    "        print(\"File does not exist at: [{}]\".format(file_loc))\n",
    "        return False\n",
    "\n",
    "def json2df(filename = \"smerp_labeled_validation\",dataset_dir=\"\"):\n",
    "    catid2cattxt_map = load_json(filename=filename,filepath=dataset_dir)\n",
    "    idxs, tweets, labels = [], [], []\n",
    "    for idx in catid2cattxt_map.keys():\n",
    "        idxs.append(idx)\n",
    "        tweets.append(catid2cattxt_map[idx][\"parsed_tweet\"])\n",
    "        labels.append(catid2cattxt_map[idx][\"classes\"])\n",
    "\n",
    "    df = pd.DataFrame.from_dict({\"idx\"    :idxs,\n",
    "                                    \"tweets\"   :tweets,\n",
    "                                    \"labels\":labels})\n",
    "    df = df[~df['tweets'].isna()]\n",
    "    df.to_csv(path_or_buf=join(dataset_dir,filename + \"_df.csv\"))\n",
    "    print(\"Data shape = {} \".format(df.shape))\n",
    "    return df\n",
    "\n",
    "df = json2df()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "from spacy.lang.en.stop_words import STOP_WORDS\n",
    "spacy_en = spacy.load(\"en_core_web_sm\")\n",
    "# Isnt required here as already cleaned dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Function to tokenize input text for cleaning\n",
    "def tokenizer_spacy(input_text: str,remove_stopwords=False):\n",
    "    input_text = spacy_en(input_text)\n",
    "    tokens = []\n",
    "    for token in input_text:\n",
    "        if remove_stopwords and token.text in STOP_WORDS:\n",
    "            continue\n",
    "        tokens.append(token.text)\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Cleaning function\n",
    "import re\n",
    "# all_stops = set(STOP_WORDS) | set(string.punctuation)\n",
    "def clean_txt(txt):\n",
    "    txt = re.sub('[^A-Za-z0-9 ]+', '', txt)\n",
    "    txt = [token for token in tokenizer_spacy(txt) if token.lower() not in STOP_WORDS]\n",
    "    txt = \" \".join(txt)\n",
    "    return txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Apply cleaning function to input data\n",
    "df[\"tweets\"] = df[\"tweets\"].apply(clean_txt)\n",
    "logger.debug(type(df[\"tweets\"].iloc[0]))\n",
    "logger.debug(df[\"tweets\"].iloc[0])\n",
    "logger.debug(df.head())\n",
    "df[\"tweets\"].str.replace('\\s{2,}', ' ')  ## Replaces multiple spaces to single\n",
    "df.to_csv(datafile,index=None,header=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(396, 3)\n",
      "(56, 3)\n",
      "(114, 3)\n"
     ]
    }
   ],
   "source": [
    "## Divide data to train, val and test\n",
    "train_size, val_size = 0.7, 0.1\n",
    "train_df = df[0:int(df.shape[0] * train_size)]\n",
    "print(train_df.shape)\n",
    "val_df = df[int(df.shape[0] * train_size):int(df.shape[0] * (train_size + val_size))]\n",
    "print(val_df.shape)\n",
    "test_df = df[int(df.shape[0] * (train_size + val_size)):int(df.shape[0])]\n",
    "print(test_df.shape)\n",
    "\n",
    "train_df.to_csv('data/Tweet_GCN_BERT_train.csv', sep=',', index=False, header=True)\n",
    "val_df.to_csv('data/Tweet_GCN_BERT_val.csv', sep=',', index=False, header=True)\n",
    "test_df.to_csv('data/Tweet_GCN_BERT_test.csv', sep=',', index=False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_spl_tokens(tweets):\n",
    "    return \"[CLS] \" + tweets + \" [SEP]\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Prepare input text for BERT and truncate long text larger than 512 tokens\n",
    "max_seq_length = 32\n",
    "txts_tokens = []\n",
    "txts_segment_ids = []\n",
    "indexed_tokens = []\n",
    "i=0\n",
    "max_len = 0\n",
    "max_tokens = []\n",
    "for txt in train_df[\"tweets\"]:\n",
    "    txt_cleaned = add_spl_tokens(txt)\n",
    "    txt_tokens = BERT_tokenizer.tokenize(add_spl_tokens(txt))\n",
    "    if len(txt_tokens) > max_seq_length:\n",
    "        i+=1\n",
    "        if len(txt_tokens) > max_len: \n",
    "            max_len = len(txt_tokens)\n",
    "            max_tokens = txt_tokens\n",
    "        txt_tokens = txt_tokens[:max_seq_length]\n",
    "    txts_tokens.append(txt_tokens)\n",
    "    txt_segment_ids = [1] * len(txt_tokens)\n",
    "    txts_segment_ids.append(txt_segment_ids)\n",
    "    txt_tokens_ids = BERT_tokenizer.convert_tokens_to_ids(txt_tokens)\n",
    "    indexed_tokens.append(txt_tokens_ids)\n",
    "    \n",
    "    padding = [0] * (max_seq_length - len(txt_tokens_ids))\n",
    "    txt_tokens_ids += padding\n",
    "#     input_mask += padding\n",
    "    txt_segment_ids += padding\n",
    "\n",
    "    assert len(txt_tokens_ids) == max_seq_length\n",
    "#     assert len(input_mask) == max_seq_length\n",
    "    assert len(txt_segment_ids) == max_seq_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Documnets with more than 32 tokens:[2]\n",
      "max_len: [35]\n",
      "max_tokens: [['[CLS]', 'earth', '##sc', '##ip', '##ly', '##mun', '##i', '23', '20', 'local', 'time', 'rain', '##ew', '##s', 'reports', '112', 'dead', 'ama', '##tric', '##e', 'alone', '37', 'dead', 'pe', '##sca', '##ra', 'del', 'tr', '##ont', '##o', 'm', '##6', 'italy', 'quake', '[SEP]']]\n"
     ]
    }
   ],
   "source": [
    "print(\"Documnets with more than {} tokens:[{}]\".format(max_seq_length,i))\n",
    "print(\"max_len: [{}]\".format(max_len))\n",
    "print(\"max_tokens: [{}]\".format(max_tokens))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for tup in zip(txts_tokens[0], indexed_tokens[0]):\n",
    "    print(tup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens_tensor = torch.tensor([indexed_tokens[0]])\n",
    "segments_tensors = torch.tensor([txts_segment_ids[0]])\n",
    "print(tokens_tensor)\n",
    "print(segments_tensors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11/07/2019 12:12:31 - INFO - pytorch_pretrained_bert.modeling -   loading archive file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased.tar.gz from cache at /home/sam/.pytorch_pretrained_bert/9c41111e2de84547a463fd39217199738d1e3deb72d4fec4399e6e241983c6f0.ae3cef932725ca7a30cdcb93fc6e09150a55e2a130ec7af63975a16c153ae2ba\n",
      "11/07/2019 12:12:31 - INFO - pytorch_pretrained_bert.modeling -   extracting archive file /home/sam/.pytorch_pretrained_bert/9c41111e2de84547a463fd39217199738d1e3deb72d4fec4399e6e241983c6f0.ae3cef932725ca7a30cdcb93fc6e09150a55e2a130ec7af63975a16c153ae2ba to temp dir /tmp/tmpvcq4nnh4\n",
      "11/07/2019 12:12:35 - INFO - pytorch_pretrained_bert.modeling -   Model config {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "BertModel(\n",
       "  (embeddings): BertEmbeddings(\n",
       "    (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "    (position_embeddings): Embedding(512, 768)\n",
       "    (token_type_embeddings): Embedding(2, 768)\n",
       "    (LayerNorm): BertLayerNorm()\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       "  (encoder): BertEncoder(\n",
       "    (layer): ModuleList(\n",
       "      (0): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): BertLayerNorm()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (1): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): BertLayerNorm()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (2): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): BertLayerNorm()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (3): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): BertLayerNorm()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (4): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): BertLayerNorm()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (5): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): BertLayerNorm()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (6): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): BertLayerNorm()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (7): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): BertLayerNorm()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (8): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): BertLayerNorm()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (9): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): BertLayerNorm()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (10): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): BertLayerNorm()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (11): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): BertLayerNorm()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (pooler): BertPooler(\n",
       "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "    (activation): Tanh()\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = BertModel.from_pretrained('bert-base-uncased')\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of documents: [396]\n",
      "Number of layers: [12]\n",
      "Number of tokens: [1]\n",
      "Number of tokens: [32]\n"
     ]
    }
   ],
   "source": [
    "all_encoded_layers = []\n",
    "for txt_tokens_ids,txt_segment_ids in zip(indexed_tokens,txts_segment_ids):\n",
    "    tokens_tensor = torch.tensor([txt_tokens_ids])\n",
    "    segments_tensors = torch.tensor([txt_segment_ids])\n",
    "    assert tokens_tensor.shape == segments_tensors.shape, \"shape does not match\"\n",
    "    with torch.no_grad():\n",
    "        encoded_layers, _ = model(tokens_tensor, segments_tensors)\n",
    "        all_encoded_layers.append(encoded_layers)\n",
    "print(\"Number of documents: [{}]\".format(len(all_encoded_layers)))\n",
    "layer_i = 0\n",
    "\n",
    "print(\"Number of layers: [{}]\".format(len(all_encoded_layers[layer_i])))\n",
    "batch_i = 0\n",
    "\n",
    "print(\"Number of tokens: [{}]\".format(len(all_encoded_layers[layer_i][batch_i])))\n",
    "token_i = 0\n",
    "\n",
    "print(\"Number of tokens: [{}]\".format(len(all_encoded_layers[layer_i][batch_i][token_i]),\"\\n\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(tokenizer.vocab.keys())[5000:5020]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "token_i = 6\n",
    "batch_i = 0\n",
    "layer_i = 6\n",
    "\n",
    "print(encoded_layers[layer_i][0].shape)\n",
    "\n",
    "vec = encoded_layers[layer_i][batch_i][token_i]\n",
    "print(vec)\n",
    "\n",
    "plt.figure(figsize=(10,10))\n",
    "plt.hist(vec, bins=200)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(len(txts_tokens[1]))\n",
    "# print(txts_tokens[1])\n",
    "# print(len(txts_tokens))\n",
    "# all_embeddings = []\n",
    "# for batch_i in range(0, len(txts_tokens)):\n",
    "#     for txt_tokens in txts_tokens:\n",
    "#         token_embeddings = []\n",
    "\n",
    "#     #     print(len(txt_tokens))\n",
    "#         for token_i in range(len(txt_tokens)):\n",
    "#             hidden_layers = [] \n",
    "#             for layer_i in range(len(encoded_layers)):\n",
    "#     #             print(layer_i,batch_i,token_i)\n",
    "#     #             print(encoded_layers[layer_i][batch_i].shape)\n",
    "#                 vec = encoded_layers[layer_i][batch_i][token_i]    \n",
    "#                 hidden_layers.append(vec)\n",
    "#             token_embeddings.append(hidden_layers)\n",
    "#         all_embeddings.append(token_embeddings)\n",
    "\n",
    "# # Sanity check the dimensions:\n",
    "# print (\"Number of tokens in sequence:\", len(token_embeddings))\n",
    "# print (\"Number of layers per token:\", len(token_embeddings[0]))\n",
    "# print (\"Number of total embeddings:\", len(all_embeddings))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# token_vecs_cat = []\n",
    "# token_vecs_sum = []\n",
    "\n",
    "# for token in token_embeddings:\n",
    "#     cat_vec = torch.cat((token[-1], token[-2], token[-3], token[-4]), 0)\n",
    "#     token_vecs_cat.append(cat_vec)\n",
    "#     sum_vec = torch.sum(torch.stack(token)[-4:], 0)\n",
    "#     token_vecs_sum.append(sum_vec)\n",
    "\n",
    "# print('Concatenated vector shape is: %d x %d' % (len(token_vecs_cat), len(token_vecs_cat[0])))\n",
    "# print('Sum vector shape is: %d x %d' % (len(token_vecs_sum), len(token_vecs_sum[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "396\n",
      "Count of sentence embeddings found: 396\n",
      "Single sentence embedding vector of shape: torch.Size([1, 768])\n"
     ]
    }
   ],
   "source": [
    "## Generate sentence lavel vector by averaging the second to last hidden layer of each token producing a single 768 length vector.\n",
    "all_sentences_embedding = []\n",
    "\n",
    "for batch_i in range(len(txts_tokens)):\n",
    "    all_sentences_embedding.append(torch.mean(all_encoded_layers[batch_i][11], 1))\n",
    "\n",
    "print(len(all_sentences_embedding))\n",
    "\n",
    "print (\"Count of sentence embeddings found:\",len(all_sentences_embedding))\n",
    "print (\"Single sentence embedding vector of shape:\",all_sentences_embedding[0].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def knn_lbl_majority(test_features,train_features,train_labels,k=5):\n",
    "\n",
    "#     from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "#     NN = KNeighborsClassifier(n_neighbors=k)\n",
    "#     NN.fit(train_features,train_labels)\n",
    "#     test_labels = NN.predict(test_features)\n",
    "#     ## To get probabilities: test_labels_probas = NN.predict_proba(test_features)\n",
    "#     return test_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def neighrest_neighbors(test_features,train_features,n,k=5):\n",
    "    from sklearn.neighbors import NearestNeighbors\n",
    "\n",
    "    NN = NearestNeighbors(n_neighbors=k)\n",
    "    NN.fit(train_features)\n",
    "    closest_neighbors_from_train = NN.kneighbors(test_features,\n",
    "    return_distance=False)\n",
    "    return closest_neighbors_from_train[0:n]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def knn_majority_label(test_features,train_features,train_labels,k=5):\n",
    "    import numpy as np\n",
    "    n=2\n",
    "    n_idxs = neighrest_neighbors(test_features, train_features, n)\n",
    "    count = np.zeros(4)\n",
    "    #n is for precision at k\n",
    "    print(test_features)\n",
    "    print(train_features)\n",
    "    print(train_labels)\n",
    "    for ii in n_idxs:\n",
    "        count = np.zeros(4)\n",
    "        for l in range(0,n):\n",
    "        for i in n_idxs[l]:\n",
    "            for j in train_labels[i]:\n",
    "                count[j] = count[j] + 1\n",
    "\n",
    "    \n",
    "    print(np.argsort(count)[4-n:])\n",
    "        \n",
    "    \n",
    "\n",
    "\n",
    "# print(test_labels[0:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_majority_label()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}